#!/bin/bash -e

#------------------------------------------------------------------------
#the script running with the condition that you should send you public key
# to the remote host with cmd "ssh-copy-id -i key-path host-ip"

HOST='120.92.77.212'
USER='ubuntu'
NOW_TIMESTAMP=`date +%s`
NOW_TIMESTAMP=$[NOW_TIMESTAMP - 1800]
DATE=`date -d @$NOW_TIMESTAMP +"%Y%m%d"`
HOUR=`date -d @$NOW_TIMESTAMP +"%H"`
OSS_PATH=/data/log/$DATE/$HOUR/
LOG_NAME='world_oss*'
PWD=`pwd`

# mysql relation
OSS_DATABASE=oss
MYSQL_USENAME='root'
MYSQL_PASSWD='unknow'
MYSQL_HOST='localhost'
MYSQL_PORT=3306

usage() {
    echo "usage: $0 -d workDir -H host -m mysqlHost -P port -u user -p password -D database -h help"
    echo "        workDir: work directory default [current path]"
    echo "        host: remote host ip default [120.92.77.212]"
    echo "        user: mysql user default [root]"
    echo "        password: mysql password"
    echo "        mysqlHost: mysql host ip default [localhost]"
    echo "        port: mysql port default [3306]"
    echo "        database: mysql database default [oss]"
    echo ""
}

while getopts "d:H:u:p:P:D:" option; do
    case "$option" in
        d)
            PWD=$OPTARG
            ;;
        H)
            HOST=$OPTARG
            ;;
        m)
            MYSQL_HOST=$OPTARG
            ;;
        u)
            MYSQL_USERNAME=$OPTARG
            ;;
        p)
            MYSQL_PASSWD=$OPTARG
            ;;
        P)
            MYSQL_PORT=$OPTARG
            ;;
        D)
            OSS_DATABASE=$OPTARG
            ;;
        h)
            usage;
            exit 0;
            ;;
        :|\?)
            usage;
            exit 1;
            ;;
    esac
done


if [ $# -le 1 ] || [ $MYSQL_PASSWD == 'unknow' ]; then
    usage
    exit 0
fi


WORK_DIR="$PWD/oss_log/$DATE/$HOUR/"
SOURCE_DIR=$WORK_DIR"source/"
CSV_DIR=$WORK_DIR"csv/"

if [ ! -d $WORK_DIR ]; then
    mkdir $WORK_DIR -p
fi

if [ ! -d $SOURCE_DIR ]; then
    mkdir $SOURCE_DIR
fi

scp $USER@$HOST:$OSS_PATH$LOG_NAME $SOURCE_DIR

rm -fr $CSV_DIR
mkdir $CSV_DIR

cd $SOURCE_DIR
for file in `ls`; do
    echo "splitting oss log file: $file..."
    awk -F'|' '{print >> "../csv/"$1".csv"}' $file
    echo "oss log file: $file has been split."
done

#ls  | xargs -i{} awk -F '|' '{print >> "../csv/"$1".csv"}' {}

cd $CSV_DIR
for file in `ls *.csv`; do
    echo "removing prefix of $file..."
    rm -f tmp.file
    cut -d'|' -f2- $file > tmp.file
    mv tmp.file $file
    echo "prefix of $file has been removed."
done

for file in `ls *.csv`; do
    OSS_TABLE=`basename $file .csv`
    echo "importing $file into database: $OSS_DATABASE, table: $OSS_TABLE..."
    mysql -u $MYSQL_USERNAME --password=$MYSQL_PASSWD -h $MYSQL_HOST -P $MYSQL_PORT --execute="LOAD DATA LOCAL INFILE \"$file\" INTO TABLE $OSS_TABLE FIELDS TERMINATED BY \"|\"; SHOW WARNINGS;" $OSS_DATABASE

    #mysql --user=$MYSQL_USERNAME --password=$MYSQL_PASSWD --host=$MYSQL_HOST --port=$MYSQL_PORT --execute="LOAD DATA LOCAL INFILE \"$file\" INTO TABLE $OSS_TABLE FIELDS TERMINATED BY \"|\"; SHOW WARNINGS;" $OSS_DATABASE
    echo "$file imported."
#    sleep 1
done

date
echo "collecting finish\n\n"
